import sys, re
from nltk import FreqDist
import instanceFactory
import cnl2mc_console
'''
sqlConfigDict = {"host":"mysql.it.usyd.edu.au",
                 "user":"swaps_hosrep",
                 "passwd":"DzTwJoir",
                 "db":"swaps_hosrep"}
'''
sqlConfigDict = {"host":"localhost",
                 "user":"root",
                 "passwd":"mysqlpw1",
                 "db":"hosrep"}
    
console = cnl2mc_console.Console()
console.config(sys.argv[1])

def randomSelectTextNode(amount, sourceNodeAmount = None, seed = 1):
    """ random select the 'amount' of text, using 'seed' """
    import random, mysqlAgent
    random.seed(1)
    ma = mysqlAgent.MysqlAgent(sqlConfigDict)
    
    wholeTextNodeDict = ma.getRandomDataNodeDict(sourceNodeAmount) # no args indicates to get all of them
    
    
    print len(wholeTextNodeDict)##<DEBUG> delete this </DEBUG>
    
    selectTextNodeList = random.sample(wholeTextNodeDict.values(), amount)
    return (selectTextNodeList,ma)

def collectMarks(text, globalMarkFreqDist):
    marks = re.findall(r"<\w+>", text)
    for mark in marks:
        globalMarkFreqDist.inc(mark)
    return
    
def main(nodeAmount, sourceNodeAmount = None):
    
    (selectTextNodeList,ma) = randomSelectTextNode(nodeAmount, sourceNodeAmount)
    
    dcNBInRepColumnStat = "requestId INT NOT NULL UNIQUE,\
                           snomedCodeSamples INT,\
                           snomedCodeTypes INT "
                           
    ma.createTableUseSchema("DcNBInRep", dcNBInRepColumnStat)
    
    dcFreqInRepColumnStat = "requestId INT NOT NULL,\
                             snomedCode CHAR(7), \
                             frequency INT "
    ma.createTableUseSchema("DcFreqInRep", dcFreqInRepColumnStat)
    
    dcFreqOverallColumnStat = "SnomedCode CHAR(7) UNIQUE,\
                               Frequency INT"
    ma.createTableUseSchema("DcFreqOverall", dcFreqOverallColumnStat)
    
    globalClassFreqDist = FreqDist()
    
    print("DcNBInRep\n")##<DEBUG> delete this </DEBUG>    
    for textNode in selectTextNodeList:
        ma.saveIntoTable("DcNBInRep", 
                         ["requestId","snomedCodeSamples","snomedCodeTypes"], 
                         [textNode.reqID, textNode.classFreqDist.N(), textNode.classFreqDist.B()])
        
        print("DcFreqInRep%s\n"%(`textNode`))##<DEBUG> delete this </DEBUG>        
        for clzFreqTuple in textNode.classFreqDist.items():
            ma.saveIntoTable("DcFreqInRep", 
                             ["requestId","snomedCode","frequency"], 
                             [textNode.reqID, clzFreqTuple[0], clzFreqTuple[1]])
        textNode.checkInClasses(globalClassFreqDist)
    
    print("DcFreqOverall\n")##<DEBUG> delete this </DEBUG>
    for clzFreqTuple in globalClassFreqDist.items():
        ma.saveIntoTable("DcFreqOverall", 
                         ["SnomedCode", "frequency"], 
                         [clzFreqTuple[0], clzFreqTuple[1]])    
    
    return    


def extractMarkups(amount = None):
    globalMarkupFreqDist = FreqDist()    
    instFact = instanceFactory.InstanceFactory.getInstance()
    ma = instFact.getMysqlAgent()
    if amount != None:
        limit = "LIMIT %d"%(amount,)
    else:
        limit = ''
    ma.cursor.execute("""                                
                        SELECT requestID, ResultText
                        FROM resulttext_table AS rtt
                        %s ORDER BY requestID;
                        """%limit)
    lastRowId = 0
    while ma.cursor.rownumber < ma.cursor.rowcount:        
        ridAndTextRow = ma.cursor.fetchone()
        ## check whether IDs are incremental by 1
        if lastRowId != (int(ridAndTextRow[0])-1):
            print "<warning> There is a gap between [RequestID %d] and [RequestID %d] </warning>\n"%(lastRowId,ridAndTextRow[0])
        collectMarks(ridAndTextRow[1], globalMarkupFreqDist)
        lastRowId = int(ridAndTextRow[0])
        del ridAndTextRow
    reportFreqDistHistogram(globalMarkupFreqDist, "../experiments/markups.txt")        
    return

def reportFreqDistHistogram(freqDist, filename, description = None):
    filename = filename+'.htm'
    try:
        f = open(filename,'w')
        f.write(str(description)+"<p/>")
        f.write("%d types, the total count is %d<br/>\n"%(freqDist.B(), freqDist.N()))
        f.write('<body><table border="1">')
        f.write("<tr><th>NAME</th><th>COUNT</th><th>FREQ</th></tr>")
        f.write('\n'.join([ "<tr><td>%s</td><td>%d</td><td>%.3f%%</td></tr>"%(k, freqDist[k] ,float(freqDist.freq(k))*100) for k in freqDist.sorted()]))
        f.write('</table></body>')
    finally:
        f.close()
    print "'%s' has been saved."%filename
    return
    

def readRequestIdList(ma, amount = None):
    if amount != None:
        limit = "LIMIT %d"%(amount)
    else:
        limit = ''
    ma.cursor.execute("""                                
                        SELECT requestID, ResultText
                        FROM resulttext_table AS rtt
                        ORDER BY requestID %s;
                        """%limit)
    IdList = []
    lastRowId = 0
    while ma.cursor.rownumber < ma.cursor.rowcount:        
        ridAndTextRow = ma.cursor.fetchone()
        ## check whether IDs are incremental by 1
        newRowId = int(ridAndTextRow[0])
        if lastRowId != (newRowId - 1):
            print "<warning> There is a gap between [RequestID %d] and [RequestID %d] </warning>\n"%(lastRowId,ridAndTextRow[0])
        lastRowId = newRowId
        IdList.append(newRowId)
        del ridAndTextRow    
    return IdList

def wordAndClassHistogram(wordsFreqDist, focusedClassList, dataNodesIter):
    # initialise a word-class matrix
    wcHistDict = {}
    occurs = '[occurs in subset]'
    total = '[sum]'
    words = wordsFreqDist.sorted()[:1000]
    for keyword in words:
        wcHistDict[keyword] = {}
        for codes in focusedClassList:
            wcHistDict[keyword][codes] = 0
    # fill values into the matrix
    for dataNode in dataNodesIter:
        for word in dataNode.wordsFreqDist:
            if word in words:
                for code in dataNode.classFreqDist:
                    if code in focusedClassList:
                        wcHistDict[word][code] += dataNode.wordsFreqDist[word]
                try:
                    wcHistDict[word][total] = sum([wcHistDict[word][c] for c in focusedClassList])
                    wcHistDict[word][occurs] = wordsFreqDist[word]
                except KeyError:
                    print "%s\t%s\n"%(c,word)
                
                
    # output the matrix into a file
    recordFilename = "../experiments/wordAndClassHistogram.txt"
    f = open(recordFilename,'w+')
    f.write("[word]\t%s\t%s\t%s\n"%('\t'.join(focusedClassList), total, occurs))
    for keyword in words:
        line = "%s\t%s\t%s\t%s\n"%(keyword,'\t'.join([str(wcHistDict[keyword][code]) for code in focusedClassList]),
                                   str(wcHistDict[keyword][total]), str(wcHistDict[keyword][occurs]))
        f.write(line)
    print "'%s' saved"%recordFilename
    f.close()
    
def histogramConceptWithId(console, isCaseSensitive):
    from ttsct_src import TokenMatcher,Sentence
    from ttsct_src.Segment import Segment
    from ttsct_src.SCTPreprocessors.DescriptionToConcept import getConcepts 
    sqlConfigDict = console.xmlConf.getSqlConfig()
    dataNodeDict = console.readTrainingDataNodeDict(sqlConfigDict)
    exprFd = FreqDist()
    didFd = FreqDist()
    cidFd = FreqDist()
    for dataNode in dataNodeDict.values():#<for:dataNone>
        if isCaseSensitive:
            lines = dataNode.textStr.split('\n')
        else:
            lines = dataNode.textStr.lower().split('\n')
        for line in lines:
            sentence = Sentence.Sentence(line)
            sentence = TokenMatcher.matchToken(sentence)
            snomedTerms = sentence.getSegmentByType(Segment.SCT_TERM)
            for t in snomedTerms:
                exprFd.inc(t.match.expression)
                didList = t.match.ids
                for i in didList:
                    didFd.inc(i)
                    cidSubList = getConcepts([i])
                    for j in cidSubList:
                        cidFd.inc(j)                  
                    print "<RID>%d</RID> <expression>%s</expression> <DID>%s</DID> <CIDS>%s</CIDS>"%(dataNode.reqID, t.match.expression, i ,','.join(cidSubList))
    #</for:dataNone>
    amount = console.xmlConf.getRandomAmount()
    reportFreqDistHistogram(exprFd, './experiments/SnomedConceptHistgram-%sIns-case%s.txt'%(len(dataNodeDict),isCaseSensitive), description = 'the "Concept" histogram from %d texts'%amount)
    reportFreqDistHistogram(didFd, './experiments/SnomedDescpIdHistgram-%sIns-case%s.txt'%(len(dataNodeDict),isCaseSensitive), description = 'the "Description ID" histogram from %d texts'%amount)
    reportFreqDistHistogram(cidFd, './experiments/SnomedConceptIdHistgram-%sIns-case%s.txt'%(len(dataNodeDict),isCaseSensitive), description = 'the "Concept ID" histogram from %d texts'%amount)
   
def histogramAllTitlesInTags(console):
    print "collecting tags..."
    tagSearcher = re.compile(r"<\w+>[\s\w]*</\w+>",re.IGNORECASE)
    #titleMatcher = re.compile(r"<title>[\s\w]*</title>",re.IGNORECASE)
    sqlConfigDict = console.xmlConf.getSqlConfig()
    dataNodeDict = console.readTrainingDataNodeDict(sqlConfigDict)
    titleFd = FreqDist()
    tagFd = FreqDist()
    titleCount = 0
    for dataNode in dataNodeDict.values():#<for:dataNone>        
        line = dataNode.textStr
        tags = tagSearcher.findall(line)
        inNodeCount = 0
        for tag in tags:##
            tagFd.inc(tag.replace('<',"&lt;").replace('>',"&gt;"))
            if tag.lower().startswith("<title>"):###
                titleFd.inc(tag.replace('<',"&lt;").replace('>',"&gt;"))
                inNodeCount += 1
                titleCount += 1 
                print "found: No.%s [%s.%s] \"%s\""%(titleCount, dataNode.reqID, inNodeCount, tag)
            ###
        ## 
    #</for:dataNone>
    amount = console.xmlConf.getRandomAmount()
    reportFreqDistHistogram(titleFd, './experiments/sections.txt', description = 'the "tags" histogram from %d texts'%amount)
    reportFreqDistHistogram(titleFd, './experiments/titles.txt', description = 'the "[Title]" histogram from %d texts'%amount)
   

if __name__ == "__main__":
    #main(10000)
    #open("../experiments/texttest.txt",'w').write('test')
    #extractMarkups()
    #readRequestIdList(1000)
    from cnl2mc_console import Console
    conf = Console()
    configFile = sys.argv[1]
    conf.config(configFile)
    #histogramConceptWithId(conf,True)
    histogramConceptWithId(conf,False)
    #histogramAllTitlesInTags(conf)
    pass
    

